package com.uxsino.simo.query;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.bouncycastle.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.uxsino.commons.logicSelector.AbstractSelector;
import com.uxsino.commons.logicSelector.GROUPBY_TYPE;
import com.uxsino.commons.logicSelector.ISelector;
import com.uxsino.commons.logicSelector.SelectorFactory;
import com.uxsino.commons.logicSelector.SelectorGroup;
import com.uxsino.commons.utils.config.IResourceWalker;
import com.uxsino.commons.utils.config.PropDocument;
import com.uxsino.commons.utils.config.PropElement;
import com.uxsino.commons.utils.config.PropJsonDocument;
import com.uxsino.commons.utils.config.PropXMLDocument;
import com.uxsino.simo.indicator.CompoundIndicator;
import com.uxsino.simo.indicator.IExprItem;
import com.uxsino.simo.indicator.Indicator;
import com.uxsino.simo.indicator.IndicatorNamespace;
import com.uxsino.simo.indicator.retractor.IValueRetractor;
import com.uxsino.simo.indicator.retractor.ValueRetractorFactory;
import com.uxsino.simo.networkentity.EntityDomain;
import com.uxsino.simo.networkentity.EntityInfo;
import com.uxsino.simo.selector.EntitySelectorContext;
import com.uxsino.simo.utils.ConfigLoadingContext;
import com.uxsino.simo.utils.ConfigPropLoader;

/**
 * Manage all queries
 * 
 *
 */
public class Querier {
    private static Logger logger = LoggerFactory.getLogger(Querier.class);

    // auto increment id for each query group instance
    private int id_key = 0;

    private ConcurrentMap<String, QueryEntry> queryGroups = new ConcurrentHashMap<>();

    private final ReentrantReadWriteLock rwlQueryGroups = new ReentrantReadWriteLock();

    private final Lock rlQueryGroups = rwlQueryGroups.readLock();

    private final Lock wlQueryGroups = rwlQueryGroups.writeLock();

    private IndicatorNamespace indicatorNS;

    private EntityDomain entityDomain;

    public Querier(EntityDomain domain, IndicatorNamespace ns) {
        indicatorNS = ns;
        entityDomain = domain;
    }

    private class QueryEntry {
        ISelector<EntityInfo> selector;

        IQueryGroup query;

        // boolean isCustom = false;

        public QueryEntry(ISelector<EntityInfo> selector, IQueryGroup query) {
            this.selector = selector;
            this.query = query;
        }
    }

    /**
     * load queries from config source
     * @param resourceWalkers see {@link IResourceWalker}
     * @return the {@link ConfigLoadingContext} contains loading results
     */
    public ConfigLoadingContext loadQueryTemplates(IResourceWalker... resourceWalkers) {
        ConfigLoadingContext lctxt = new ConfigLoadingContext(IndicatorNamespace.class);
        Arrays.stream(resourceWalkers).filter(Objects::nonNull).forEach(walker -> {
            try {
                walker.forEach(url -> {
                    String srcName = url.getFile();
                    logger.info("load query file:{}", srcName);
                    lctxt.setCurrentSource(srcName);
                    InputStream is = null;

                    try {
                        is = url.openStream();
                        loadFromXmlStream(is, srcName, lctxt);
                    } catch (IllegalArgumentException e) {
                        lctxt.error("", "error loading indicator");
                    } catch (IOException e) {
                        lctxt.error("", "error reading indicator file", e);
                    } finally {
                        if (is != null) {
                            try {
                                is.close();
                            } catch (IOException e) {
                            }
                        }
                    }
                });
            } catch (IOException e) {
                logger.error("error loading indicators", e);
            }
        });
        return lctxt;

    }

    /**
     * load a query from Json 
     */
    public void loadFromJson(JsonNode node, ConfigLoadingContext lctxt) {
        PropJsonDocument doc = new PropJsonDocument(node);
        loadQuery(doc, node.get("id").asText(), lctxt);

    }

    public void loadCustomQuery(JsonNode queries, ConfigLoadingContext lctxt) {
        if (queries == null || queries.size() == 0) {
            return;
        }
        if (queries.isArray()) {
            for (JsonNode jsonNode : queries) {
                loadFromJson(jsonNode, lctxt);
            }
        } else {
            loadFromJson(queries, lctxt);
        }
    }

    /**
     * load a query from {@link InputStream} of xml 
     * @param is xml stream
     * @param srcName
     * @param lctxt loading context
     */
    public void loadFromXmlStream(InputStream is, String srcName, ConfigLoadingContext lctxt) {
        PropXMLDocument doc = new PropXMLDocument(is, srcName);
        if (doc.isValid()) {
            loadQuery(doc, null, lctxt);
        } else {
            lctxt.error("0", "error reading query file.");
        }
    }

    /**
     * load a query from xml {@link File}
     * @param xmlFile
     * @param queryId
     * @param lctxt loading context
     */
    public void loadFromXml(File xmlFile, String queryId, ConfigLoadingContext lctxt) {
        PropXMLDocument doc = new PropXMLDocument(xmlFile, queryId);
        if (doc.isValid()) {
            loadQuery(doc, queryId, lctxt);
        } else {
            lctxt.error("0", "error reading query file.");
        }
    }

    /**
     * load a query from doc {@link PropDocument}
     * @param doc
     * @param queryId
     * @param lctxt loading context
     */
    public void loadQuery(PropDocument doc, String queryId, ConfigLoadingContext lctxt) {

        PropElement eSelector = doc.getFirstElement("selector");
        PropElement eQueries = doc.getFirstElement("queries");

        if (eQueries == null) {
            lctxt.error("0", "cannot find queries node");
            return;
        }

        SelectorFactory<EntityInfo> sf = new SelectorFactory<EntityInfo>();
        for (PropElement eQuery : eQueries.getElements("query")) {
            String protocolNames = eQuery.getProp("protocol");
            if (StringUtils.isBlank(protocolNames)) {
                continue;
            }
            for (String protocolName : protocolNames.split(",")) {
                SelectorGroup<EntityInfo> selector = sf.createSelectorGroup(GROUPBY_TYPE.All, null);
                if (eSelector != null) {
                    selector.setSourceInfo(eSelector.getSourceName(), eSelector.getSourceLocation());
                    selector.add(sf.createSelectorGroup(eSelector));
                }
                if (protocolName.length() > 0 && !"none".equals(protocolName)) {
                    protocolName = Strings.toLowerCase(protocolName);
                    selector.add(sf.createPropSelector("protocols" + "[" + protocolName + "]", "exists", ""));
                }
                selector.solveReference(entityDomain::getEntitySelector);
                QueryTemplate qt = new QueryTemplate(newQueryGroupId());
                lctxt.addObject("" + qt.getId(), eQuery.getSourceLocation());
                try {
                    qt.setProtocolName(protocolName);
                    lctxt.getPropLoader().loadProperties(qt, eQuery);
                    loadParameters(qt, eQuery, lctxt);
                    loadReplaces(qt, eQuery);
                    loadRetracts(qt, eQuery, lctxt);
                    qt.setSourceInfo(eQuery.getSourceName(), eQuery.getSourceLocation());
                    if (StringUtils.isEmpty(queryId)) {
                        queryGroups.put(UUID.randomUUID().toString(), new QueryEntry(selector, qt));
                    } else {
                        qt.isCustom = true;
                        queryGroups.put("customer_query_"+queryId/*abandoned* + UUID.randomUUID().toString()*/, new QueryEntry(selector, qt));
                    }
                } catch (Exception e) {
                    lctxt.error(eQuery.getSourceLocation(), "error loading {}.", queryId, e);
                }
            }
        }

        for (PropElement eCombine : eQueries.getElements("combine")) {
            String itemIndicatorName = eCombine.getProp("indicator");
            Indicator itemIndicator = indicatorNS.getIndicator(itemIndicatorName);
            if (itemIndicator == null) {
                lctxt.error(eCombine.getSourceLocation(), "indicator {} not found.", itemIndicatorName);
                continue;
            }
            String[] protocolNames = { null };
            if (!StringUtils.isBlank(eCombine.getProp("protocol"))) {
                protocolNames = eCombine.getProp("protocol").split(",");
            }
            for (String protocolName : protocolNames) {
                SelectorGroup<EntityInfo> selector = sf.createSelectorGroup(GROUPBY_TYPE.All, null);
                if (eSelector != null) {
                    selector.setSourceInfo(eSelector.getSourceName(), eSelector.getSourceLocation());
                    selector.add(sf.createSelectorGroup(eSelector));
                }
                selector.solveReference(entityDomain::getEntitySelector);

                try {
                    IQueryCombine<?, ?> combine = loadCombine(protocolName, eCombine, selector, eSelector,
                        itemIndicator, lctxt);
                    combine.setSourceInfo(eCombine.getSourceName(), eCombine.getSourceLocation());
                    if (StringUtils.isEmpty(queryId)) {
                        queryGroups.put(UUID.randomUUID().toString(), new QueryEntry(selector, combine));
                    } else {
                        queryGroups.put(queryId, new QueryEntry(selector, combine));
                    }
                } catch (Exception e) {
                    lctxt.error(eCombine.getSourceLocation(), "error loading combine. {}", e);
                }
            }
        }
    }

    private IQueryCombine<?, ?> loadCombine(String protocolName, PropElement eCombine,
        SelectorGroup<EntityInfo> selector, PropElement eSelector, Indicator itemIndicator,
        ConfigLoadingContext lctxt) {
        ConfigPropLoader loader = new ConfigPropLoader(lctxt);
        String itemIndicatorName = eCombine.getProp("indicator");
        SelectorFactory<EntityInfo> sf = new SelectorFactory<EntityInfo>();

        @SuppressWarnings("unchecked")
        IQueryCombine<?, ?> combine = (IQueryCombine<CompoundIndicator, List<Map<String, Object>>>) QueryCombine
            .create(eCombine.getProp("method"), itemIndicator);
        if(combine != null){
            try {
                combine.loadProp(eCombine, lctxt);
            }catch (Exception e) {
                logger.warn("[combine indicator error ]: {}", e);
            }
        }

        for (PropElement eQuery : eCombine.getElements("query")) {
            QueryTemplate qt = loadQuery(protocolName, eQuery, eSelector, loader, lctxt);
            protocolName = qt.getProtocolName();
            if (qt != null) {
                if (qt.retractsIndicator(itemIndicator)) {
                    if (protocolName.length() > 0) {
                        selector.add(sf.createPropSelector("protocols" + "[" + protocolName + "]", "exists", ""));
                    }
                    combine.addQuery(qt);
                } else {
                    lctxt.error(eCombine.getSourceLocation(), "sub query doesn't retract indicator {}",
                        itemIndicatorName);
                }
            }
        }

        combine.setSourceInfo(eCombine.getSourceName(), eCombine.getSourceLocation());
        return combine;
    }

    private QueryTemplate loadQuery(String protocolName, PropElement eQuery, PropElement eSelector,
        ConfigPropLoader loader, ConfigLoadingContext lctxt) {
        if (protocolName == null) {
            protocolName = eQuery.getProp("protocol");
        }
        QueryTemplate qt = new QueryTemplate(newQueryGroupId());
        lctxt.addObject("" + qt.getId(), eQuery.getSourceLocation());
        try {
            qt.setProtocolName(protocolName);
            loader.loadProperties(qt, eQuery);

            if (qt instanceof IExprItem) {
                lctxt.getEvaluator().compileExprItem((IExprItem) qt);
            }
            loadParameters(qt, eQuery, lctxt);
            loadReplaces(qt, eQuery);
            loadRetracts(qt, eQuery, lctxt);
        } catch (Exception e) {
            lctxt.error(eQuery.getSourceLocation(), "error loading query.", e);
            return null;
        }

        qt.setSourceInfo(eQuery.getSourceName(), eQuery.getSourceLocation());
        return qt;
    }

    private void loadReplaces(QueryTemplate qt, PropElement eQuery) {
        List<PropElement> nReplaces = eQuery.getElements("replace");
        Map<String, String> replaces = new HashMap<>();

        for (PropElement eReplace : nReplaces) {
            replaces.putIfAbsent(eReplace.getProp("from"), eReplace.getProp("to"));
        }

        qt.addReplaces(replaces);

    }

    private void loadParameters(QueryTemplate qt, PropElement eQuery, ConfigLoadingContext lctxt) {
        ConfigPropLoader loader = new ConfigPropLoader(lctxt);
        PropElement eParameters = eQuery.getFirstElement("parameters");

        if (eParameters == null) {
            return;
        }

        for (PropElement eParam : eParameters.getElements("parameter")) {
            String pName = eParam.getProp("name");

            if (StringUtils.isEmpty(pName)) {
                lctxt.error(eParam.getSourceLocation(), "parameter name not found, ignored. ");
                continue;
            }
            String indName = eParam.getProp("indicator");

            if (StringUtils.isEmpty(indName)) {
                // maybe there are parameter types other than indicator
                // reference in the future
                // then change this...
                lctxt.error(eParam.getSourceLocation(), "indicator name not found, ignored. ");
                continue;
            }

            Indicator indParam = indicatorNS.getIndicator(indName);

            if (indParam == null) {
                lctxt.error(eParam.getSourceLocation(), "indicator {} not found, ignored.", indName);
                continue;

            }
            String fieldName = eParam.getProp("field");

            IndicatorRefQueryParameter qp = new IndicatorRefQueryParameter();
            qp.setName(pName);
            qp.setIndicator(indParam, fieldName);
            try {
                loader.loadProperties(qp, eParam);
            } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
                lctxt.error(eParam.getSourceLocation(), "error loading parameter");
            }
            qt.addParameter(pName, qp);
        }
    }

    private void loadRetracts(QueryTemplate qt, PropElement eQuery, ConfigLoadingContext lctxt) {

        PropElement eRetracts = eQuery.getFirstElement("retracts");
        if (eRetracts == null) {
            return;
        }
        for (PropElement eRetractor : eRetracts.getElements("retract")) {
            String indicatorName = eRetractor.getProp("indicator");
            Indicator ind = indicatorNS.getIndicator(indicatorName);
            if (indicatorName == null || ind == null) {
                lctxt.error(eQuery.getSourceLocation(), "Indicator: {} is not defined. Retractor ignored",
                    indicatorName);
                continue;
            }
            IValueRetractor vr;
            String parserName = eRetractor.getProp("parser");

            try {
                if (StringUtils.isEmpty(parserName)) {
                    parserName = ProtocolManager.getDefaultParserName(qt.getProtocolName());
                }

                vr = ValueRetractorFactory.createValueRetractor(parserName, ind.getIndicatorType());

            } catch (Exception e1) {
                lctxt.error(eQuery.getSourceLocation(), "error creating parser ({}) for {}.",
                    eRetractor.getProp("parser"), ind.name, e1);
                continue;
            }

            try {
                lctxt.getPropLoader().loadProperties(vr, eRetractor);
                vr.loadProp(ind, eRetractor, lctxt);

                if (vr instanceof IExprItem) {
                    lctxt.getEvaluator().compileExprItem((IExprItem) vr);
                }
                qt.addRetractor(ind, vr);
            } catch (Exception e) {
                lctxt.error(eRetractor.getSourceLocation(), "error loading retractor.", e);
            }
        }

    }

    public IndicatorNamespace getNamespace() {
        return indicatorNS;
    }

    public EntityDomain getEntityDomain() {
        return entityDomain;
    }

    /**
     * get a list of all queries
     * @return list of queries
     */
    public List<IQueryGroup> dumpQueries() {
        rlQueryGroups.lock();
        ArrayList<IQueryGroup> r = new ArrayList<>();
        queryGroups.forEach((key, value) -> {
            r.add(value.query);
        });
        rlQueryGroups.unlock();
        return r;
    }

    // return queryId:selectorInfo_query_type
    public Map<String, String> findQueriesByIndicatorAndProtocol(String indName, String protocolName) {
        Map<String, String> queryInfoMap = new HashMap<>();
        rlQueryGroups.lock();

        queryGroups.forEach((key, value) -> {
            List<QueryTemplate> qtList = new ArrayList<>();
            if (value.query.getQueryType().equals(IQueryGroup.QUERY_TYPE_COMBINE)) {
                qtList = ((QueryCombine<?, ?>) (value.query)).getQueries();
            } else if (value.query.getQueryType().equals(IQueryGroup.QUERY_TYPE_QUERY)) {
                qtList = new ArrayList<>();
                qtList.add((QueryTemplate) value.query);
            }
            boolean found = false;
            for (QueryTemplate qtItem : qtList) {
                if (qtItem.getProtocolName().equals(protocolName)) {
                    Set<Indicator> indSet = qtItem.getRetractors().keySet();
                    for (Indicator ind : indSet) {
                        if (ind.name.equals(indName)) {
                            found = true;
                            break;
                        }
                    }
                }
                if (found){
                    break;
                }
            }
            if (found) {
                queryInfoMap.put(key, "selector source info:" + ((AbstractSelector<?>) value.selector).getSourceInfo()
                        + " " + value.selector.toXml() + " query type: " + value.query.getQueryType());
            }
        });
        rlQueryGroups.unlock();
        return queryInfoMap;
    }

    public void deleteAllCustom() {
        wlQueryGroups.lock();
        try {
            queryGroups.forEach((key, value) -> {
                if (value.query.isCustom()) {
                    queryGroups.remove(key, value);
                }
            });
        } finally {
            wlQueryGroups.unlock();
        }
    }

    public void deleteQuery(String queryId) {
        wlQueryGroups.lock();
        try {
            queryGroups.remove(queryId);
            invalidateFeasibleCache();
        } finally {
            wlQueryGroups.unlock();
        }
    }

    /** 
     * count the queries
     * @return count
     */
    public int getQueryCount() {
        return queryGroups.size();
    }

    private Cache<Pair<String, String>, IQueryGroup[]> feasibleQueryCache = CacheBuilder.newBuilder().maximumSize(10000)
        .expireAfterAccess(20, TimeUnit.MINUTES).build();

    private boolean cacheing = false;

    /**
     * fetch queries matches given ne and retracts indicator
     * 
     * @param ne the network entity
     * @param indicator which indicator to search
     * @return list of queries found
     */
    public IQueryGroup[] getFeasibleQueries(EntityInfo ne, Indicator indicator) {
        if (cacheing) {
            Pair<String, String> key = Pair.of(ne.id, indicator.name);
            IQueryGroup[] fq = feasibleQueryCache.getIfPresent(key);
            if (fq == null || fq.length == 0) {
                fq = getFeasibleQueriesNoCache(ne, indicator);
                for (IQueryGroup iQueryGroup : fq) {
                    if (iQueryGroup.isCustom()) {
                        return fq;
                    }
                }
                feasibleQueryCache.put(key, fq);
            }
            return fq;
        }
        return getFeasibleQueriesNoCache(ne, indicator);

    }

    /**
     * cach feasible query for follow-up search
     */
    public void startCachingFeasible() {
        cacheing = true;
    }

    /**
     * stop caching feasible query. for query updating
     */
    public void stopCachingFeasible() {
        feasibleQueryCache.invalidateAll();
        cacheing = false;
    }

    /**
     * invalidate the feasible query cach (when queries are updated)
     */
    public void invalidateFeasibleCache() {
        feasibleQueryCache.invalidateAll();
    }

    /**
     * force a new search of feasible query ignoring cached results
     * @param ne
     * @param indicator
     * @return
     */
    public IQueryGroup[] getFeasibleQueriesNoCache(EntityInfo ne, Indicator indicator) {

        EntitySelectorContext context = new EntitySelectorContext(entityDomain);
        context.setObject(ne);
        return getFeasibleQueries(indicator, context);
    }

    /**
     * fetch queries matches given {@link EntityInfo} and retracts indicator. set {@link EntityInfo} in context
     * first.
     * 
     * @param indicator
     * @param context
     *            use context to track the searching process.
     * @return an array of feasible queries in descending order of priority. see {@link IQueryGroup.getPriority}
     */
    public IQueryGroup[] getFeasibleQueries(Indicator indicator, EntitySelectorContext context) {
        ArrayList<IQueryGroup> queries = new ArrayList<>();
        for (QueryEntry entry : queryGroups.values()) {
            context.setTestFor(entry.query);
            if (entry.query.retractsIndicator(indicator) && entry.selector.accept(context)) {
                queries.add(entry.query);
            }
        }

        if (queries.size() > 0) {
            queries.sort((x, y) -> {
                // Descending order by priority
                return y.getPriority() - x.getPriority();
            });
        }
        return queries.toArray(new IQueryGroup[queries.size()]);
    }

    // increase id_key and return the new id
    public synchronized int newQueryGroupId() {
        id_key++;
        return id_key;
    }

}
